import redis
import requests
import random

def should_request(url):
    client = redis.Redis(host='localhost', port=6379, db=0)
    urls = client.lrange('requested_urls', 0, -1)
    return url.encode() in urls

def save_request_record(url):
    client = redis.Redis(host='localhost', port=6379, db=0)
    client.lpush('requested_urls',url)
    client.close()

def main():
    urls = [f'https://httpbin.org/get?age={random.randint(1, 100)}' for i in range(1000)]
    urls = set(urls)
    for url in urls:
        if not should_request(url):
            res = requests.get(url)
            print(res)
            save_request_record(url)
        else:
            print("不需要爬取")

main()